""""""
from string import whitespace, punctuation

import numpy as np
import pandas
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from zhon.hanzi import punctuation as zh_punctuation


def demo():
    df = pandas.read_json("1.json")
    df2 = pandas.DataFrame()
    df2["words"] = df["remarks"].apply(
        lambda v: "".join([w for w in v if w not in whitespace + punctuation + zh_punctuation]))
    business_type = set(df["business_type"])
    for t in business_type:
        df2[t] = df["business_type"].apply(lambda v: int(v == t))
    df2.to_excel("原始标注.xlsx")

    df3 = pandas.read_excel("原始标注.xlsx", 0, index_col=0)

    label_cols = df3.columns[2:]
    print(label_cols)

    train, test = train_test_split(df3)
    lens = train.words.str.len()
    print(lens.mean(), lens.std(), lens.max())

    COMMENT = 'words'
    train[COMMENT].fillna("unknown", inplace=True)
    test[COMMENT].fillna("unknown", inplace=True)

    vec = TfidfVectorizer(
        ngram_range=(1, 2), min_df=3, max_df=0.9, strip_accents='unicode', use_idf=1, smooth_idf=1, sublinear_tf=1
    )
    trn_term_doc = vec.fit_transform(train[COMMENT])
    test_term_doc = vec.transform(test[COMMENT])
    print(trn_term_doc, test_term_doc)

    def pr(y_i, y):
        p = x[y == y_i].sum(0)
        return (p + 1) / ((y == y_i).sum() + 1)

    x = trn_term_doc
    test_x = test_term_doc

    def get_mdl(y):
        y = y.values
        r = np.log(pr(1, y) / pr(0, y))
        m = LogisticRegression(C=4, dual=False)
        x_nb = x.multiply(r)
        return m.fit(x_nb, y), r

    preds = np.zeros((len(test), len(label_cols)))

    for i, j in enumerate(label_cols):
        print('fit', j)
        m, r = get_mdl(train[j])
        preds[:, i] = m.predict_proba(test_x.multiply(r))[:, 1]



if __name__ == '__main__':
    demo()
